{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient descent for other functions\n",
    "\n",
    "Gradient descent works well for other functions, even if they are non-convex; gradient descent, if it converges, converges to a local minimum. Lets look at the function $$f(a,b) = \\sin(a)\\cos(b).$$ We would like to find parameters $(a,b)$ that minimize the function $f$. \n",
    "\n",
    "The gradients are:\n",
    "$$\\frac{\\partial f}{\\partial a} = \\cos(a)\\cos(b),$$\n",
    "$$\\frac{\\partial f}{\\partial b} = -\\sin(a)\\sin(b).$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code source: Sebastian Curi and Andreas Krause.\n",
    "\n",
    "# Python Notebook Commands\n",
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# Numerical Libraries\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from matplotlib import rcParams\n",
    "rcParams['figure.figsize'] = (10, 5)\n",
    "\n",
    "# IPython Libraries\n",
    "import IPython\n",
    "import ipywidgets\n",
    "from ipywidgets import interact, interactive, interact_manual\n",
    "\n",
    "\n",
    "# Custom Libraries\n",
    "from utilities.load_data import polynomial_data\n",
    "from utilities import plot_helpers\n",
    "from utilities.util import gradient_descent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class sincos(object):\n",
    "    structured = False\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    @property\n",
    "    def number_samples(self):\n",
    "        return 0\n",
    "\n",
    "    def loss(self, w, *args):\n",
    "        return np.sin(w[0]) * np.cos(w[1])\n",
    "\n",
    "    def gradient(self, w, *args):\n",
    "        return np.array([np.cos(w[0]) * np.cos(w[1]), -np.sin(w[0]) * np.sin(w[1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rcParams['figure.figsize'] = (10, 5)   # Change this if figures look ugly. \n",
    "rcParams['font.size'] = 16   # Change this if figures look ugly. \n",
    "\n",
    "eta_widget = ipywidgets.FloatSlider(value=4, min=1e-1, max=10, step=1e-1, readout_format='.1f', \n",
    "                                    description='Learning rate:', style={'description_width': 'initial'}, \n",
    "                                    continuous_update=False)\n",
    "n_iter_widget = ipywidgets.IntSlider(value=10, min=5, max=50, step=1, description='Number of iterations:',\n",
    "                                     style={'description_width': 'initial'}, continuous_update=False)\n",
    "\n",
    "lr_widget = ipywidgets.RadioButtons(options=['Bold driver', 'AdaGrad', 'Annealing', 'None'], value='None',\n",
    "                                    description='Learning rate heuristics:', style={'description_width': 'initial'})\n",
    "\n",
    "a0_widget = ipywidgets.FloatSlider(value=-0.9, min=-3, max=3, step=.1, readout_format='.1f', description='w_0:',\n",
    "                                   style={'description_width': 'initial'}, continuous_update=False)\n",
    "\n",
    "b0_widget = ipywidgets.FloatSlider(value=1.1, min=-3, max=3, step=.1, readout_format='.1f', description='w_1:',\n",
    "                                   style={'description_width': 'initial'}, continuous_update=False)\n",
    "def optimize_sincos(eta, n_iter, learning_rate_scheduling, a0, b0):\n",
    "    loss_function = sincos()\n",
    "    w0 = np.array([a0, b0])\n",
    "    \n",
    "    if learning_rate_scheduling == 'None':\n",
    "        learning_rate_scheduling = None\n",
    "\n",
    "    opts = {'eta0': eta,\n",
    "            'n_iter': n_iter,\n",
    "            'learning_rate_scheduling': learning_rate_scheduling\n",
    "            }\n",
    "    trajectory, _ = gradient_descent(w0, loss_function, opts=opts)\n",
    "    contourplot = plt.subplot(111)\n",
    "    dataplot = None\n",
    "    contour_opts = {'x_label': '$w_0$', 'y_label': '$w_1$', 'title': 'Weight trajectory', 'legend': False}\n",
    "    plot_opts = {'contour_opts': contour_opts}\n",
    "\n",
    "    plot_helpers.linear_regression_progression(np.array([]), np.array([]), trajectory, np.array([]), \n",
    "                                               loss_function.loss,\n",
    "                                               contourplot, dataplot, options=plot_opts)\n",
    "\n",
    "\n",
    "interact_manual(optimize_sincos, eta=eta_widget, n_iter=n_iter_widget, learning_rate_scheduling=lr_widget,\n",
    "                a0=a0_widget, b0=b0_widget);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}